{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import roc_auc_score, auc\n",
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import confusion_matrix,classification_report,accuracy_score,precision_score,recall_score,f1_score,roc_auc_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import hamming_loss\n",
    "from sklearn.metrics import roc_curve, auc\n",
    "from sklearn.metrics import accuracy_score\n",
    "from scipy import interp\n",
    "\n",
    "from sklearn import preprocessing\n",
    "from sklearn.preprocessing import MultiLabelBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorflow\\python\\framework\\dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\requests\\__init__.py:80: RequestsDependencyWarning: urllib3 (1.26.8) or chardet (3.0.4) doesn't match a supported version!\n",
      "  RequestsDependencyWarning)\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\tensorboard\\compat\\tensorflow_stub\\dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras.layers import *\n",
    "from keras.optimizers import *\n",
    "from keras.losses import binary_crossentropy\n",
    "from keras.metrics import *\n",
    "from keras import callbacks\n",
    "from keras.callbacks import EarlyStopping\n",
    "from sklearn.metrics import roc_auc_score, auc, precision_recall_curve, confusion_matrix\n",
    "import numpy as np\n",
    "import sklearn.metrics as metrics\n",
    "from collections import Counter\n",
    "import pandas as pd\n",
    "import random\n",
    "from sklearn.model_selection import KFold\n",
    "from tqdm import tqdm\n",
    "import scipy.sparse as sp\n",
    "from copy import deepcopy\n",
    "import warnings \n",
    "import os\n",
    "from sklearn import preprocessing\n",
    "\n",
    "from sklearn.preprocessing import MultiLabelBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(directory):\n",
    "    D_SSM = np.loadtxt(directory + '/D_SM.txt')\n",
    "    M_FSM = np.loadtxt(directory + '/M_SM.txt')\n",
    "\n",
    "\n",
    "\n",
    "    ID = np.zeros(shape=(D_SSM.shape[0], D_SSM.shape[1]))\n",
    "    IM = np.zeros(shape=(M_FSM.shape[0], M_FSM.shape[1]))\n",
    "    for i in range(D_SSM.shape[0]):\n",
    "        for j in range(D_SSM.shape[1]):\n",
    "            if D_SSM[i][j] == 0:\n",
    "                ID[i][j] = D_SSM[i][j]######D_GSM[i][j]\n",
    "            else:\n",
    "                ID[i][j] = D_SSM[i][j]\n",
    "    for i in range(M_FSM.shape[0]):\n",
    "        for j in range(M_FSM.shape[1]):\n",
    "            if M_FSM[i][j] == 0:\n",
    "                IM[i][j] = M_FSM[i][j]#######M_GSM[i][j]\n",
    "            else:\n",
    "                IM[i][j] = M_FSM[i][j]\n",
    "                \n",
    "    ID = pd.DataFrame(ID).reset_index()\n",
    "    IM = pd.DataFrame(IM).reset_index()\n",
    "    ID.rename(columns = {'index':'id'}, inplace = True)\n",
    "    IM.rename(columns = {'index':'id'}, inplace = True)\n",
    "    ID['id'] = ID['id'] + 1\n",
    "    IM['id'] = IM['id'] + 1\n",
    "    \n",
    "    return ID, IM\n",
    "\n",
    "def sample(directory, random_seed):\n",
    "    all_associations = pd.read_csv(directory + '/drug_mutation_pairs.csv', names=['Drug', 'Mutation', 'label'])\n",
    "    known_associations = all_associations.loc[all_associations['label'] == 1]\n",
    "    known_associations_resistance=all_associations.loc[all_associations['label'] == -1]\n",
    "    unknown_associations = all_associations.loc[all_associations['label'] == 0]\n",
    "\n",
    "    random_negative = unknown_associations.sample(n=known_associations.shape[0]+known_associations_resistance.shape[0], random_state=random_seed, axis=0)\n",
    "\n",
    "    sample_df = known_associations.append(known_associations_resistance.append(random_negative))\n",
    "    print('sample_df',sample_df)\n",
    "\n",
    "    sample_df.reset_index(drop=True, inplace=True)\n",
    "   \n",
    "    return sample_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def performances(class_arr,y_true, y_pred, y_prob):\n",
    "\n",
    "    \n",
    "    \n",
    "    y_true_prob=class_arr.reshape(-1)\n",
    "    y_pred_prob = y_prob.reshape(-1)\n",
    "\n",
    "    fpr, tpr, _ = roc_curve(y_true_prob, y_pred_prob)\n",
    "    roc_auc = auc(fpr, tpr)\n",
    "    \n",
    "    pr, re, _ = precision_recall_curve(y_true_prob, y_pred_prob)\n",
    "    aupr = auc(re, pr)\n",
    "    \n",
    "    accuracy=accuracy_score(y_true, y_pred)\n",
    "    f1=f1_score(y_true, y_pred, average=\"macro\")\n",
    "    recall=recall_score(y_true, y_pred, average='macro')\n",
    "    precision=precision_score(y_true, y_pred, average='macro')\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "  \n",
    "    print('acc={:.4f}|precision={:.4f}|recall={:.4f}|f1={:.4f}|auc={:.4f}|aupr={:.4f}'.format(accuracy, precision, recall, f1, roc_auc,aupr))\n",
    "    return (y_true, y_pred, y_prob), (accuracy, precision, recall, f1, roc_auc, aupr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def obtain_data(directory, isbalance):\n",
    "\n",
    "    ID, IM = load_data(directory)\n",
    "\n",
    "    if isbalance:\n",
    "        dtp = sample(directory, random_seed = 1234)\n",
    "    else:\n",
    "        dtp = pd.read_csv(directory + '/drug_mutation_pairs.csv', names=['Drug', 'Mutation', 'label'])\n",
    "\n",
    "    mirna_ids = list(set(dtp['Drug']))\n",
    "    disease_ids = list(set(dtp['Mutation']))\n",
    "    random.shuffle(mirna_ids)\n",
    "    random.shuffle(disease_ids)\n",
    "    print('# Drug = {} | Mutation = {}'.format(len(mirna_ids), len(disease_ids)))\n",
    "\n",
    "    mirna_test_num = int(len(mirna_ids) / 5)\n",
    "    disease_test_num = int(len(disease_ids) / 5)\n",
    "    print('# Test: Drug = {} | Mutation = {}'.format(mirna_test_num, disease_test_num))    \n",
    "    \n",
    "    samples = pd.merge(pd.merge(dtp, ID, left_on = 'Drug', right_on = 'id'), IM, left_on = 'Mutation', right_on = 'id')\n",
    "\n",
    "    samples.drop(labels = ['id_x', 'id_y'], axis = 1, inplace = True)\n",
    "\n",
    "    return ID, IM, dtp, mirna_ids, disease_ids, mirna_test_num, disease_test_num, samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_task_Tp_train_test_idx(samples):\n",
    "    kf = KFold(n_splits = 5, shuffle = True, random_state = 1234)\n",
    "\n",
    "    train_index_all, test_index_all, n = [], [], 0\n",
    "    train_id_all, test_id_all = [], []\n",
    "    fold = 0\n",
    "    for train_idx, test_idx in tqdm(kf.split(samples.iloc[:, 3:])): \n",
    "        print('-------Fold ', fold)\n",
    "        train_index_all.append(train_idx) \n",
    "        test_index_all.append(test_idx)\n",
    "\n",
    "        train_id_all.append(np.array(dtp.iloc[train_idx][['Drug', 'Mutation', 'label']]))\n",
    "        test_id_all.append(np.array(dtp.iloc[test_idx][['Drug', 'Mutation', 'label']]))\n",
    "        #print('train_index_all',train_index_all)\n",
    "        #print('test_index_all',test_index_all)\n",
    "\n",
    "        print('# Pairs: Train = {} | Test = {}'.format(len(train_idx), len(test_idx)))\n",
    "        fold += 1\n",
    "    #print('train_id_all',train_id_all)   \n",
    "    return train_index_all, test_index_all, train_id_all, test_id_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_task_Tm_Td_train_test_idx(item, ids, dtp):\n",
    "    \n",
    "    test_num = int(len(ids) / 5)\n",
    "    \n",
    "    train_index_all, test_index_all = [], []\n",
    "    train_id_all, test_id_all = [], []\n",
    "    \n",
    "    for fold in range(5):\n",
    "        print('-------Fold ', fold)\n",
    "        if fold != 4:\n",
    "            test_ids = ids[fold * test_num : (fold + 1) * test_num]\n",
    "        else:\n",
    "            test_ids = ids[fold * test_num :]\n",
    "\n",
    "        train_ids = list(set(ids) ^ set(test_ids))\n",
    "        print('# {}: Train = {} | Test = {}'.format(item, len(train_ids), len(test_ids)))\n",
    "\n",
    "        test_idx = dtp[dtp[item].isin(test_ids)].index.tolist()\n",
    "        train_idx = dtp[dtp[item].isin(train_ids)].index.tolist()\n",
    "        random.shuffle(test_idx)\n",
    "        random.shuffle(train_idx)\n",
    "        print('# Pairs: Train = {} | Test = {}'.format(len(train_idx), len(test_idx)))\n",
    "        assert len(train_idx) + len(test_idx) == len(dtp)\n",
    "\n",
    "        train_index_all.append(train_idx) \n",
    "        test_index_all.append(test_idx)\n",
    "        \n",
    "        train_id_all.append(train_ids)\n",
    "        test_id_all.append(test_ids)\n",
    "        #print('test_id_all.append(test_ids)',test_id_all.append(test_ids))\n",
    "        \n",
    "    return train_index_all, test_index_all, train_id_all, test_id_all"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 随机森林"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_rf(train_index_all, test_index_all, samples):\n",
    "    \n",
    "    fold = 0\n",
    "    for train_idx, test_idx in zip(train_index_all, test_index_all):\n",
    "        print('-----------------------Fold = ', str(fold))\n",
    "\n",
    "        X = samples.iloc[:, 3:]\n",
    "        y = samples['label']\n",
    "\n",
    "        \n",
    "        scaler = preprocessing.MinMaxScaler().fit(X.iloc[train_idx,:])\n",
    "        X = scaler.transform(X)\n",
    "\n",
    "        print('X',X.shape)\n",
    "\n",
    "        x_train, y_train = X[train_idx], y[train_idx]\n",
    "        x_test, y_test = X[test_idx], y[test_idx]\n",
    "        \n",
    "\n",
    "        print('x_test',x_test)\n",
    "        print('x_test.shape',x_test.shape)\n",
    "\n",
    "        clf = RandomForestClassifier(random_state =19961231)\n",
    "        clf.fit(x_train, y_train)\n",
    "\n",
    "        df=pd.DataFrame(y_train, columns=['label'])\n",
    "        b=df['label'].tolist()\n",
    "        NewList = [[x] for x in b]\n",
    "        cf = MultiLabelBinarizer()\n",
    "        class_arr=cf.fit_transform(NewList)\n",
    "        print('class_arr',class_arr)\n",
    "\n",
    "        ###test  标签\n",
    "        \n",
    "        df_test=pd.DataFrame(y_test, columns=['label'])\n",
    "        b_test=df_test['label'].tolist()\n",
    "        NewList_test = [[x] for x in b_test]\n",
    "        cf_test = MultiLabelBinarizer()\n",
    "        class_arr_test=cf.fit_transform(NewList_test)\n",
    "        \n",
    "        print('df_test',df_test)\n",
    "        print('df_test',df_test.shape)\n",
    "    \n",
    "        \n",
    "        y_train_prob = clf.predict_proba(x_train)\n",
    "        y_test_prob = clf.predict_proba(x_test)\n",
    "        #print('x_train',x_train)\n",
    "        #print('y_train_prob',y_train_prob)\n",
    "        \n",
    "        y_train_pred = clf.predict(x_train)\n",
    "        y_test_pred = clf.predict(x_test)\n",
    "        print('y_train_prob',y_train_prob)\n",
    "        print('y_train_prob',y_train_prob.shape)\n",
    "        print('y_test_prob',y_test_prob)\n",
    "        print('y_test_prob',y_test_prob.shape)\n",
    "        #print('y_train_pred',y_train_pred)\n",
    "        #print('y_test_pred',y_test_pred)\n",
    "\n",
    "        #print('y_train',y_train)\n",
    "        print('Train:')\n",
    "        ys_train, metrics_train = performances(class_arr,y_train, y_train_pred, y_train_prob)\n",
    "        #ys_train, metrics_train = performances(y_train, y_train_pred, y_train_prob[:, 1])\n",
    "        #ys_train, metrics_train = performances(y_train, y_train_pred, y_train_prob[:, 0])\n",
    "        \n",
    "        print('Test:')\n",
    "        ys_test, metrics_test = performances(class_arr_test,y_test, y_test_pred, y_test_prob)\n",
    "        #ys_test, metrics_test = performances(y_test, y_test_pred, y_test_prob[:, 1])\n",
    "        #ys_test, metrics_test = performances(y_test, y_test_pred, y_test_prob[:, 0])\n",
    "\n",
    "        fold += 1\n",
    "    \n",
    "    return ys_train, metrics_train, ys_test, metrics_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5it [00:00, 385.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sample_df         Drug  Mutation  label\n",
      "153        1       154      1\n",
      "1188       2       528      1\n",
      "1772       3       451      1\n",
      "2920       5       277      1\n",
      "3207       5       564      1\n",
      "...      ...       ...    ...\n",
      "4878       8       252      0\n",
      "20521     32        31      0\n",
      "118974   180       656      0\n",
      "12639     20        81      0\n",
      "66290    101       191      0\n",
      "\n",
      "[3508 rows x 3 columns]\n",
      "# Drug = 184 | Mutation = 661\n",
      "# Test: Drug = 36 | Mutation = 132\n",
      "========== isbalance = True | task = Tp\n",
      "-------Fold  0\n",
      "# Pairs: Train = 2806 | Test = 702\n",
      "-------Fold  1\n",
      "# Pairs: Train = 2806 | Test = 702\n",
      "-------Fold  2\n",
      "# Pairs: Train = 2806 | Test = 702\n",
      "-------Fold  3\n",
      "# Pairs: Train = 2807 | Test = 701\n",
      "-------Fold  4\n",
      "# Pairs: Train = 2807 | Test = 701\n",
      "-----------------------Fold =  0\n",
      "X (3508, 845)\n",
      "x_test [[0.         0.89851304 0.97045899 ... 0.560056   0.69342503 0.65846584]\n",
      " [0.95066492 0.99188569 0.99521619 ... 0.560056   0.69342503 0.65846584]\n",
      " [0.99368461 0.90122084 0.964258   ... 0.560056   0.69342503 0.65846584]\n",
      " ...\n",
      " [0.99224646 0.93913923 0.98992068 ... 0.0095598  0.91164888 0.91771088]\n",
      " [0.98708627 0.91567048 0.96353454 ... 0.14406849 0.96616978 0.94602872]\n",
      " [0.9599187  0.8054221  0.89525815 ... 0.98781382 0.13328406 0.11471283]]\n",
      "x_test.shape (702, 845)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "class_arr [[0 0 1]\n",
      " [0 0 1]\n",
      " [0 0 1]\n",
      " ...\n",
      " [0 1 0]\n",
      " [1 0 0]\n",
      " [1 0 0]]\n",
      "df_test       label\n",
      "0         1\n",
      "1         1\n",
      "2         1\n",
      "11        1\n",
      "13        1\n",
      "...     ...\n",
      "3472     -1\n",
      "3480      0\n",
      "3494      1\n",
      "3498     -1\n",
      "3505      0\n",
      "\n",
      "[702 rows x 1 columns]\n",
      "df_test (702, 1)\n",
      "y_train_prob [[0.02 0.08 0.9 ]\n",
      " [0.01 0.15 0.84]\n",
      " [0.03 0.2  0.77]\n",
      " ...\n",
      " [0.07 0.92 0.01]\n",
      " [0.88 0.08 0.04]\n",
      " [0.78 0.19 0.03]]\n",
      "y_train_prob (2806, 3)\n",
      "y_test_prob [[0.03 0.38 0.59]\n",
      " [0.32 0.11 0.57]\n",
      " [0.08 0.13 0.79]\n",
      " ...\n",
      " [0.07 0.38 0.55]\n",
      " [0.51 0.23 0.26]\n",
      " [0.04 0.91 0.05]]\n",
      "y_test_prob (702, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6667|precision=0.6180|recall=0.6183|f1=0.6133|auc=0.8305|aupr=0.7190\n",
      "-----------------------Fold =  1\n",
      "X (3508, 845)\n",
      "x_test [[0.98052622 0.96020941 0.99174189 ... 0.560056   0.69342503 0.65846584]\n",
      " [0.94001478 0.80196109 0.87985237 ... 0.560056   0.69342503 0.65846584]\n",
      " [0.99061974 0.8594114  0.94167571 ... 0.560056   0.69342503 0.65846584]\n",
      " ...\n",
      " [0.9804817  0.96914205 0.999644   ... 0.07936482 0.72447728 0.75032233]\n",
      " [0.85370926 0.65283357 0.76285083 ... 0.0095598  0.91164888 0.91771088]\n",
      " [0.90931547 0.7746458  0.85438094 ... 0.58620025 0.6706326  0.63616927]]\n",
      "x_test.shape (702, 845)\n",
      "class_arr [[0 0 1]\n",
      " [0 0 1]\n",
      " [0 0 1]\n",
      " ...\n",
      " [0 1 0]\n",
      " [1 0 0]\n",
      " [1 0 0]]\n",
      "df_test       label\n",
      "4         1\n",
      "6         1\n",
      "9         1\n",
      "20        1\n",
      "21        1\n",
      "...     ...\n",
      "3464     -1\n",
      "3488     -1\n",
      "3491     -1\n",
      "3496      0\n",
      "3499      0\n",
      "\n",
      "[702 rows x 1 columns]\n",
      "df_test (702, 1)\n",
      "y_train_prob [[0.02 0.09 0.89]\n",
      " [0.23 0.01 0.76]\n",
      " [0.   0.05 0.95]\n",
      " ...\n",
      " [0.03 0.95 0.02]\n",
      " [0.95 0.04 0.01]\n",
      " [0.87 0.12 0.01]]\n",
      "y_train_prob (2806, 3)\n",
      "y_test_prob [[0.07 0.38 0.55]\n",
      " [0.12 0.24 0.64]\n",
      " [0.05 0.21 0.74]\n",
      " ...\n",
      " [0.54 0.38 0.08]\n",
      " [0.07 0.51 0.42]\n",
      " [0.04 0.3  0.66]]\n",
      "y_test_prob (702, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6296|precision=0.5803|recall=0.5627|f1=0.5657|auc=0.7897|aupr=0.6443\n",
      "-----------------------Fold =  2\n",
      "X (3508, 845)\n",
      "x_test [[0.95740277 0.89551491 0.94262958 ... 0.560056   0.69342503 0.65846584]\n",
      " [0.98077867 0.95938347 0.99297059 ... 0.560056   0.69342503 0.65846584]\n",
      " [0.98960492 0.8766412  0.94751095 ... 0.560056   0.69342503 0.65846584]\n",
      " ...\n",
      " [0.99224646 0.93913923 0.98992068 ... 0.02063822 0.83941842 0.85883865]\n",
      " [0.98708627 0.91567048 0.96353454 ... 0.0095598  0.91164888 0.91771088]\n",
      " [0.94284629 0.81475128 0.8864809  ... 0.87409143 0.01952924 0.02868409]]\n",
      "x_test.shape (702, 845)\n",
      "class_arr [[0 0 1]\n",
      " [0 0 1]\n",
      " [0 0 1]\n",
      " ...\n",
      " [0 1 0]\n",
      " [1 0 0]\n",
      " [1 0 0]]\n",
      "df_test       label\n",
      "17        1\n",
      "18        1\n",
      "24        1\n",
      "34        1\n",
      "36        1\n",
      "...     ...\n",
      "3485      1\n",
      "3490     -1\n",
      "3492      1\n",
      "3495      1\n",
      "3501      0\n",
      "\n",
      "[702 rows x 1 columns]\n",
      "df_test (702, 1)\n",
      "y_train_prob [[0.02 0.11 0.87]\n",
      " [0.16 0.04 0.8 ]\n",
      " [0.01 0.06 0.93]\n",
      " ...\n",
      " [0.   0.99 0.01]\n",
      " [0.93 0.05 0.02]\n",
      " [0.8  0.18 0.02]]\n",
      "y_train_prob (2806, 3)\n",
      "y_test_prob [[0.45 0.14 0.41]\n",
      " [0.11 0.38 0.51]\n",
      " [0.04 0.09 0.87]\n",
      " ...\n",
      " [0.55 0.12 0.33]\n",
      " [0.71 0.1  0.19]\n",
      " [0.02 0.33 0.65]]\n",
      "y_test_prob (702, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6396|precision=0.6051|recall=0.5941|f1=0.5952|auc=0.8172|aupr=0.7002\n",
      "-----------------------Fold =  3\n",
      "X (3508, 845)\n",
      "x_test [[9.44457547e-01 9.85006059e-01 9.85781726e-01 ... 5.60055995e-01\n",
      "  6.93425031e-01 6.58465840e-01]\n",
      " [9.72201365e-01 8.86966500e-01 9.45673982e-01 ... 5.60055995e-01\n",
      "  6.93425031e-01 6.58465840e-01]\n",
      " [9.50895036e-01 9.87015753e-01 9.92536607e-01 ... 5.60055995e-01\n",
      "  6.93425031e-01 6.58465840e-01]\n",
      " ...\n",
      " [9.80481703e-01 9.69142055e-01 9.99644003e-01 ... 8.58735000e-04\n",
      "  9.17581030e-01 9.30948646e-01]\n",
      " [9.90309290e-01 8.83879938e-01 9.62166240e-01 ... 8.74091427e-01\n",
      "  1.95292400e-02 2.86840910e-02]\n",
      " [9.78011699e-01 9.73949238e-01 9.95905899e-01 ... 4.69052069e-01\n",
      "  7.72388518e-01 7.40568395e-01]]\n",
      "x_test.shape (701, 845)\n",
      "class_arr [[0 0 1]\n",
      " [0 0 1]\n",
      " [0 0 1]\n",
      " ...\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [1 0 0]]\n",
      "df_test       label\n",
      "5         1\n",
      "7         1\n",
      "8         1\n",
      "12        1\n",
      "22        1\n",
      "...     ...\n",
      "3482      1\n",
      "3487     -1\n",
      "3489     -1\n",
      "3502     -1\n",
      "3507     -1\n",
      "\n",
      "[701 rows x 1 columns]\n",
      "df_test (701, 1)\n",
      "y_train_prob [[0.   0.16 0.84]\n",
      " [0.23 0.04 0.73]\n",
      " [0.02 0.1  0.88]\n",
      " ...\n",
      " [0.02 0.97 0.01]\n",
      " [0.02 0.97 0.01]\n",
      " [0.89 0.1  0.01]]\n",
      "y_train_prob (2807, 3)\n",
      "y_test_prob [[0.13 0.5  0.37]\n",
      " [0.21 0.17 0.62]\n",
      " [0.04 0.68 0.28]\n",
      " ...\n",
      " [0.49 0.39 0.12]\n",
      " [0.96 0.03 0.01]\n",
      " [0.6  0.34 0.06]]\n",
      "y_test_prob (701, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6205|precision=0.5783|recall=0.5636|f1=0.5609|auc=0.7924|aupr=0.6528\n",
      "-----------------------Fold =  4\n",
      "X (3508, 845)\n",
      "x_test [[0.91837205 0.99923891 0.9807622  ... 0.560056   0.69342503 0.65846584]\n",
      " [0.98374915 0.95260868 0.9886871  ... 0.560056   0.69342503 0.65846584]\n",
      " [0.98462025 0.92745375 0.97196896 ... 0.560056   0.69342503 0.65846584]\n",
      " ...\n",
      " [0.94992626 0.98949193 0.99694098 ... 0.98154657 0.01728065 0.01471034]\n",
      " [0.88191783 0.68962526 0.79216197 ... 0.98781382 0.13328406 0.11471283]\n",
      " [0.9780117  0.97394924 0.9959059  ... 0.98781382 0.13328406 0.11471283]]\n",
      "x_test.shape (701, 845)\n",
      "class_arr [[0 0 1]\n",
      " [0 0 1]\n",
      " [0 0 1]\n",
      " ...\n",
      " [1 0 0]\n",
      " [0 1 0]\n",
      " [1 0 0]]\n",
      "df_test       label\n",
      "3         1\n",
      "10       -1\n",
      "15        1\n",
      "16        1\n",
      "23        1\n",
      "...     ...\n",
      "3497      0\n",
      "3500     -1\n",
      "3503      1\n",
      "3504      0\n",
      "3506     -1\n",
      "\n",
      "[701 rows x 1 columns]\n",
      "df_test (701, 1)\n",
      "y_train_prob [[0.01 0.22 0.77]\n",
      " [0.17 0.05 0.78]\n",
      " [0.02 0.04 0.94]\n",
      " ...\n",
      " [0.99 0.   0.01]\n",
      " [0.04 0.93 0.03]\n",
      " [0.86 0.12 0.02]]\n",
      "y_train_prob (2807, 3)\n",
      "y_test_prob [[0.02 0.16 0.82]\n",
      " [0.11 0.04 0.85]\n",
      " [0.07 0.29 0.64]\n",
      " ...\n",
      " [0.05 0.76 0.19]\n",
      " [0.03 0.86 0.11]\n",
      " [0.87 0.1  0.03]]\n",
      "y_test_prob (701, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6633|precision=0.6321|recall=0.6102|f1=0.6141|auc=0.8183|aupr=0.6973\n",
      "========== isbalance = True | task = Tm\n",
      "-------Fold  0\n",
      "# Drug: Train = 148 | Test = 36\n",
      "# Pairs: Train = 2649 | Test = 859\n",
      "-------Fold  1\n",
      "# Drug: Train = 148 | Test = 36\n",
      "# Pairs: Train = 2886 | Test = 622\n",
      "-------Fold  2\n",
      "# Drug: Train = 148 | Test = 36\n",
      "# Pairs: Train = 2854 | Test = 654\n",
      "-------Fold  3\n",
      "# Drug: Train = 148 | Test = 36\n",
      "# Pairs: Train = 2721 | Test = 787\n",
      "-------Fold  4\n",
      "# Drug: Train = 144 | Test = 40\n",
      "# Pairs: Train = 2922 | Test = 586\n",
      "-----------------------Fold =  0\n",
      "X (3508, 845)\n",
      "x_test [[0.91078704 0.99511638 0.97858543 ... 0.75096257 0.50129843 0.46901191]\n",
      " [0.98238021 0.82640414 0.91792742 ... 0.85530153 0.01660413 0.03040633]\n",
      " [0.98013296 0.93926796 0.97258409 ... 0.10490446 0.99539369 0.98720432]\n",
      " ...\n",
      " [0.95342876 0.98471472 0.98904349 ... 0.03964966 0.79766639 0.82030544]\n",
      " [0.9093706  0.99699861 0.97331942 ... 0.00333833 0.89910373 0.91177676]\n",
      " [0.96801532 0.94987232 0.97870971 ... 0.12044793 0.99092008 0.980208  ]]\n",
      "x_test.shape (859, 845)\n",
      "class_arr [[1 0 0]\n",
      " [1 0 0]\n",
      " [0 1 0]\n",
      " ...\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]]\n",
      "df_test       label\n",
      "113       0\n",
      "1283      0\n",
      "2510      1\n",
      "1217      0\n",
      "33        1\n",
      "...     ...\n",
      "878       0\n",
      "1202      0\n",
      "818      -1\n",
      "2850      0\n",
      "383       0\n",
      "\n",
      "[859 rows x 1 columns]\n",
      "df_test (859, 1)\n",
      "y_train_prob [[0.86 0.07 0.07]\n",
      " [0.86 0.1  0.04]\n",
      " [0.   0.98 0.02]\n",
      " ...\n",
      " [0.   0.97 0.03]\n",
      " [0.29 0.71 0.  ]\n",
      " [0.05 0.91 0.04]]\n",
      "y_train_prob (2649, 3)\n",
      "y_test_prob [[0.41 0.48 0.11]\n",
      " [0.04 0.39 0.57]\n",
      " [0.31 0.34 0.35]\n",
      " ...\n",
      " [0.73 0.25 0.02]\n",
      " [0.07 0.74 0.19]\n",
      " [0.09 0.63 0.28]]\n",
      "y_test_prob (859, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6356|precision=0.5975|recall=0.5762|f1=0.5807|auc=0.8037|aupr=0.6716\n",
      "-----------------------Fold =  1\n",
      "X (3508, 845)\n",
      "x_test [[0.83732576 0.66465994 0.75842356 ... 0.00170895 0.92560175 0.94042724]\n",
      " [0.97390469 0.97479019 0.         ... 0.07571473 0.99985547 0.99816593]\n",
      " [0.90170381 0.         0.97479076 ... 0.12500376 0.99020662 0.9797648 ]\n",
      " ...\n",
      " [0.95464948 0.96648628 0.97931431 ... 0.9034671  0.30028356 0.27226414]\n",
      " [0.99611967 0.90439048 0.96820617 ... 0.90433029 0.29896292 0.27144442]\n",
      " [0.96475346 0.9643078  0.98396746 ... 0.02314328 0.98610195 0.98887026]]\n",
      "x_test.shape (622, 845)\n",
      "class_arr [[1 0 0]\n",
      " [1 0 0]\n",
      " [0 1 0]\n",
      " ...\n",
      " [0 1 0]\n",
      " [0 0 1]\n",
      " [1 0 0]]\n",
      "df_test       label\n",
      "2117      1\n",
      "183       1\n",
      "178       0\n",
      "226       1\n",
      "1818      0\n",
      "...     ...\n",
      "228       0\n",
      "1916     -1\n",
      "2672      1\n",
      "2031      1\n",
      "2054     -1\n",
      "\n",
      "[622 rows x 1 columns]\n",
      "df_test (622, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_train_prob [[0.96 0.02 0.02]\n",
      " [0.9  0.08 0.02]\n",
      " [0.   0.99 0.01]\n",
      " ...\n",
      " [0.04 0.8  0.16]\n",
      " [0.16 0.04 0.8 ]\n",
      " [0.82 0.08 0.1 ]]\n",
      "y_train_prob (2886, 3)\n",
      "y_test_prob [[0.27 0.57 0.16]\n",
      " [0.05 0.66 0.29]\n",
      " [0.06 0.8  0.14]\n",
      " ...\n",
      " [0.84 0.12 0.04]\n",
      " [0.07 0.49 0.44]\n",
      " [0.14 0.78 0.08]]\n",
      "y_test_prob (622, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6254|precision=0.5715|recall=0.5475|f1=0.5483|auc=0.7971|aupr=0.6557\n",
      "-----------------------Fold =  2\n",
      "X (3508, 845)\n",
      "x_test [[0.95045142 0.79196069 0.88206627 ... 0.0230039  0.8360868  0.85668815]\n",
      " [0.96700455 0.90016227 0.95035791 ... 0.98050724 0.02816016 0.02971166]\n",
      " [0.98706487 0.93605081 0.98462048 ... 0.90941104 0.00925557 0.02129169]\n",
      " ...\n",
      " [0.99299618 0.87970684 0.96137128 ... 0.95748515 0.20887417 0.18663875]\n",
      " [0.95982487 0.79900804 0.89368417 ... 0.07277218 0.74070534 0.76871312]\n",
      " [0.99224646 0.93913923 0.98992068 ... 0.86999729 0.34817062 0.31720169]]\n",
      "x_test.shape (654, 845)\n",
      "class_arr [[1 0 0]\n",
      " [1 0 0]\n",
      " [0 0 1]\n",
      " ...\n",
      " [0 1 0]\n",
      " [1 0 0]\n",
      " [0 0 1]]\n",
      "df_test       label\n",
      "556       0\n",
      "2475      1\n",
      "830      -1\n",
      "440       0\n",
      "3479      1\n",
      "...     ...\n",
      "975      -1\n",
      "2408      0\n",
      "3120      0\n",
      "708       1\n",
      "1543      1\n",
      "\n",
      "[654 rows x 1 columns]\n",
      "df_test (654, 1)\n",
      "y_train_prob [[0.94 0.05 0.01]\n",
      " [0.95 0.04 0.01]\n",
      " [0.11 0.07 0.82]\n",
      " ...\n",
      " [0.02 0.91 0.07]\n",
      " [0.84 0.08 0.08]\n",
      " [0.06 0.26 0.68]]\n",
      "y_train_prob (2854, 3)\n",
      "y_test_prob [[0.18 0.77 0.05]\n",
      " [0.1  0.19 0.71]\n",
      " [0.84 0.12 0.04]\n",
      " ...\n",
      " [0.14 0.8  0.06]\n",
      " [0.19 0.66 0.15]\n",
      " [0.1  0.6  0.3 ]]\n",
      "y_test_prob (654, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6789|precision=0.6365|recall=0.6345|f1=0.6338|auc=0.8286|aupr=0.6956\n",
      "-----------------------Fold =  3\n",
      "X (3508, 845)\n",
      "x_test [[0.94445755 0.98500606 0.98578173 ... 0.98433094 0.1468779  0.12856283]\n",
      " [0.95740277 0.89551491 0.94262958 ... 0.06166057 0.77463505 0.80645373]\n",
      " [0.97180686 0.96966646 0.98727136 ... 0.11041882 0.99394097 0.98537866]\n",
      " ...\n",
      " [0.97892051 0.82839524 0.91457023 ... 0.98718867 0.12116177 0.09969145]\n",
      " [0.9237102  0.76958984 0.85369665 ... 0.0232383  0.83296484 0.85328917]\n",
      " [0.99224646 0.93913923 0.98992068 ... 0.89345978 0.31730992 0.29041771]]\n",
      "x_test.shape (787, 845)\n",
      "class_arr [[1 0 0]\n",
      " [1 0 0]\n",
      " [0 0 1]\n",
      " ...\n",
      " [1 0 0]\n",
      " [1 0 0]\n",
      " [0 1 0]]\n",
      "df_test       label\n",
      "1547      0\n",
      "740      -1\n",
      "465       0\n",
      "1996     -1\n",
      "2025     -1\n",
      "...     ...\n",
      "771      -1\n",
      "1490      1\n",
      "3064      1\n",
      "2771      0\n",
      "517       0\n",
      "\n",
      "[787 rows x 1 columns]\n",
      "df_test (787, 1)\n",
      "y_train_prob [[0.95 0.04 0.01]\n",
      " [0.93 0.02 0.05]\n",
      " [0.01 0.24 0.75]\n",
      " ...\n",
      " [0.9  0.   0.1 ]\n",
      " [0.97 0.01 0.02]\n",
      " [0.02 0.95 0.03]]\n",
      "y_train_prob (2721, 3)\n",
      "y_test_prob [[0.55 0.24 0.21]\n",
      " [0.85 0.08 0.07]\n",
      " [0.13 0.86 0.01]\n",
      " ...\n",
      " [0.94 0.   0.06]\n",
      " [0.06 0.84 0.1 ]\n",
      " [0.08 0.01 0.91]]\n",
      "y_test_prob (787, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.5985|precision=0.5426|recall=0.5428|f1=0.5350|auc=0.7872|aupr=0.6481\n",
      "-----------------------Fold =  4\n",
      "X (3508, 845)\n",
      "x_test [[9.67004549e-01 9.00162268e-01 9.50357906e-01 ... 9.90497894e-01\n",
      "  1.24884248e-01 1.10243834e-01]\n",
      " [9.84389003e-01 9.12690370e-01 9.70538376e-01 ... 9.76149359e-01\n",
      "  1.67885191e-01 1.48554662e-01]\n",
      " [9.84389003e-01 9.12690370e-01 9.70538376e-01 ... 6.15450000e-04\n",
      "  9.21252897e-01 9.34298399e-01]\n",
      " ...\n",
      " [9.56515349e-01 9.87053546e-01 9.96323079e-01 ... 3.67111480e-02\n",
      "  9.66075257e-01 9.69377147e-01]\n",
      " [9.58702884e-01 9.83845022e-01 9.98550215e-01 ... 6.77920423e-01\n",
      "  3.61629466e-01 3.67282363e-01]\n",
      " [9.78011699e-01 9.73949238e-01 9.95905899e-01 ... 9.31348860e-02\n",
      "  7.04564421e-01 7.32247400e-01]]\n",
      "x_test.shape (586, 845)\n",
      "class_arr [[1 0 0]\n",
      " [0 1 0]\n",
      " [1 0 0]\n",
      " ...\n",
      " [0 0 1]\n",
      " [0 0 1]\n",
      " [1 0 0]]\n",
      "df_test       label\n",
      "2068     -1\n",
      "2217      0\n",
      "896       0\n",
      "2554      0\n",
      "1831      1\n",
      "...     ...\n",
      "2544      0\n",
      "2085     -1\n",
      "3192     -1\n",
      "2012      0\n",
      "107       0\n",
      "\n",
      "[586 rows x 1 columns]\n",
      "df_test (586, 1)\n",
      "y_train_prob [[0.82 0.05 0.13]\n",
      " [0.   0.9  0.1 ]\n",
      " [0.91 0.07 0.02]\n",
      " ...\n",
      " [0.01 0.11 0.88]\n",
      " [0.03 0.05 0.92]\n",
      " [0.71 0.27 0.02]]\n",
      "y_train_prob (2922, 3)\n",
      "y_test_prob [[0.29 0.48 0.23]\n",
      " [0.16 0.78 0.06]\n",
      " [0.06 0.64 0.3 ]\n",
      " ...\n",
      " [0.4  0.51 0.09]\n",
      " [0.   0.99 0.01]\n",
      " [0.43 0.54 0.03]]\n",
      "y_test_prob (586, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6177|precision=0.5857|recall=0.5697|f1=0.5703|auc=0.7963|aupr=0.6636\n",
      "========== isbalance = True | task = Td\n",
      "-------Fold  0\n",
      "# Mutation: Train = 529 | Test = 132\n",
      "# Pairs: Train = 2793 | Test = 715\n",
      "-------Fold  1\n",
      "# Mutation: Train = 529 | Test = 132\n",
      "# Pairs: Train = 2833 | Test = 675\n",
      "-------Fold  2\n",
      "# Mutation: Train = 529 | Test = 132\n",
      "# Pairs: Train = 2776 | Test = 732\n",
      "-------Fold  3\n",
      "# Mutation: Train = 529 | Test = 132\n",
      "# Pairs: Train = 2816 | Test = 692\n",
      "-------Fold  4\n",
      "# Mutation: Train = 528 | Test = 133\n",
      "# Pairs: Train = 2814 | Test = 694\n",
      "-----------------------Fold =  0\n",
      "X (3508, 845)\n",
      "x_test [[0.99412616 0.86870686 0.94691842 ... 0.98655389 0.12231043 0.10063072]\n",
      " [0.99276049 0.88252711 0.96075782 ... 0.16656606 0.97509611 0.96132091]\n",
      " [0.94445755 0.98500606 0.98578173 ... 0.98206772 0.02118915 0.01974696]\n",
      " ...\n",
      " [0.91655906 0.75248829 0.8368459  ... 0.08142929 0.86326455 0.87455024]\n",
      " [0.95740277 0.89551491 0.94262958 ... 0.07277218 0.74070534 0.76871312]\n",
      " [0.88191783 0.68962526 0.79216197 ... 0.22395135 0.94797592 0.9294508 ]]\n",
      "x_test.shape (715, 845)\n",
      "class_arr [[0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " ...\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 0 1]]\n",
      "df_test       label\n",
      "419       0\n",
      "1474     -1\n",
      "271       1\n",
      "1780      1\n",
      "1182      0\n",
      "...     ...\n",
      "558       0\n",
      "2851      1\n",
      "2555      0\n",
      "703      -1\n",
      "153       0\n",
      "\n",
      "[715 rows x 1 columns]\n",
      "df_test (715, 1)\n",
      "y_train_prob [[0.03 0.89 0.08]\n",
      " [0.02 0.9  0.08]\n",
      " [0.   0.93 0.07]\n",
      " ...\n",
      " [0.01 0.75 0.24]\n",
      " [0.01 0.88 0.11]\n",
      " [0.01 0.1  0.89]]\n",
      "y_train_prob (2793, 3)\n",
      "y_test_prob [[0.39 0.57 0.04]\n",
      " [0.19 0.35 0.46]\n",
      " [0.27 0.35 0.38]\n",
      " ...\n",
      " [0.76 0.15 0.09]\n",
      " [0.8  0.19 0.01]\n",
      " [0.34 0.55 0.11]]\n",
      "y_test_prob (715, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6490|precision=0.6076|recall=0.5929|f1=0.5965|auc=0.8088|aupr=0.6762\n",
      "-----------------------Fold =  1\n",
      "X (3508, 845)\n",
      "x_test [[0.97892051 0.82839524 0.91457023 ... 0.07065082 0.99676349 0.98989248]\n",
      " [0.87283667 0.9923895  0.9605871  ... 0.76819273 0.47826191 0.46405641]\n",
      " [0.98706487 0.93605081 0.98462048 ... 0.99884395 0.06206158 0.0505892 ]\n",
      " ...\n",
      " [0.99611967 0.90439048 0.96820617 ... 0.01746912 0.98066555 0.98403245]\n",
      " [0.94445755 0.98500606 0.98578173 ... 0.23329377 0.51651072 0.54723462]\n",
      " [0.98077867 0.95938347 0.99297059 ... 0.04802761 0.99768824 0.99816744]]\n",
      "x_test.shape (675, 845)\n",
      "class_arr [[0 1 0]\n",
      " [0 0 1]\n",
      " [0 0 1]\n",
      " ...\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]]\n",
      "df_test       label\n",
      "2157     -1\n",
      "2521      0\n",
      "3335      1\n",
      "1987      1\n",
      "86        1\n",
      "...     ...\n",
      "2437      1\n",
      "2795      0\n",
      "483       0\n",
      "1466      0\n",
      "296       0\n",
      "\n",
      "[675 rows x 1 columns]\n",
      "df_test (675, 1)\n",
      "y_train_prob [[0.04 0.58 0.38]\n",
      " [0.   0.25 0.75]\n",
      " [0.04 0.02 0.94]\n",
      " ...\n",
      " [0.1  0.77 0.13]\n",
      " [0.   0.96 0.04]\n",
      " [0.05 0.89 0.06]]\n",
      "y_train_prob (2833, 3)\n",
      "y_test_prob [[0.58 0.18 0.24]\n",
      " [0.04 0.69 0.27]\n",
      " [0.38 0.45 0.17]\n",
      " ...\n",
      " [0.05 0.7  0.25]\n",
      " [0.39 0.43 0.18]\n",
      " [0.09 0.39 0.52]]\n",
      "y_test_prob (675, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6237|precision=0.5914|recall=0.5644|f1=0.5706|auc=0.8003|aupr=0.6684\n",
      "-----------------------Fold =  2\n",
      "X (3508, 845)\n",
      "x_test [[9.55565666e-01 9.82990132e-01 9.94289559e-01 ... 9.38038707e-01\n",
      "  2.49681000e-04 4.07207400e-03]\n",
      " [8.88117561e-01 6.92146987e-01 7.92179590e-01 ... 4.03531730e-02\n",
      "  7.96037645e-01 8.18311356e-01]\n",
      " [9.74792526e-01 9.74547572e-01 9.99161348e-01 ... 4.57061623e-01\n",
      "  7.68830563e-01 7.66704655e-01]\n",
      " ...\n",
      " [9.56099354e-01 9.77475773e-01 9.81652011e-01 ... 9.85203906e-01\n",
      "  6.41874820e-02 5.15982700e-02]\n",
      " [9.54649484e-01 9.66486281e-01 9.79314307e-01 ... 9.97680041e-01\n",
      "  9.49676030e-02 8.10061910e-02]\n",
      " [9.92760495e-01 8.82527112e-01 9.60757822e-01 ... 1.12886441e-01\n",
      "  9.92396802e-01 9.82444602e-01]]\n",
      "x_test.shape (732, 845)\n",
      "class_arr [[0 1 0]\n",
      " [1 0 0]\n",
      " [1 0 0]\n",
      " ...\n",
      " [0 0 1]\n",
      " [1 0 0]\n",
      " [0 1 0]]\n",
      "df_test       label\n",
      "1709      0\n",
      "3381      0\n",
      "908       1\n",
      "382       0\n",
      "3123      0\n",
      "...     ...\n",
      "1023      0\n",
      "897       1\n",
      "1761      1\n",
      "445      -1\n",
      "947       0\n",
      "\n",
      "[732 rows x 1 columns]\n",
      "df_test (732, 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_train_prob [[0.32 0.63 0.05]\n",
      " [0.83 0.06 0.11]\n",
      " [0.86 0.14 0.  ]\n",
      " ...\n",
      " [0.06 0.27 0.67]\n",
      " [0.92 0.07 0.01]\n",
      " [0.01 0.76 0.23]]\n",
      "y_train_prob (2776, 3)\n",
      "y_test_prob [[0.07 0.78 0.15]\n",
      " [0.22 0.03 0.75]\n",
      " [0.12 0.02 0.86]\n",
      " ...\n",
      " [0.16 0.51 0.33]\n",
      " [0.69 0.3  0.01]\n",
      " [0.3  0.65 0.05]]\n",
      "y_test_prob (732, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6339|precision=0.5717|recall=0.5824|f1=0.5707|auc=0.7989|aupr=0.6724\n",
      "-----------------------Fold =  3\n",
      "X (3508, 845)\n",
      "x_test [[8.45818988e-01 6.62288565e-01 7.61479100e-01 ... 4.96810822e-01\n",
      "  7.48061116e-01 7.36228172e-01]\n",
      " [9.73958879e-01 9.68018267e-01 9.92913422e-01 ... 8.29432000e-04\n",
      "  9.21116435e-01 9.32177887e-01]\n",
      " [9.93083022e-01 8.62788554e-01 9.44815515e-01 ... 2.48168100e-03\n",
      "  9.24963978e-01 9.40942286e-01]\n",
      " ...\n",
      " [9.68832511e-01 8.22880216e-01 9.07407611e-01 ... 2.69222282e-01\n",
      "  6.68373253e-01 7.18109976e-01]\n",
      " [8.68068386e-01 6.77510987e-01 7.77021061e-01 ... 9.91580353e-01\n",
      "  1.22504082e-01 1.06250099e-01]\n",
      " [9.71123263e-01 9.71808813e-01 9.94632952e-01 ... 6.05125000e-04\n",
      "  9.30683964e-01 9.43081584e-01]]\n",
      "x_test.shape (692, 845)\n",
      "class_arr [[0 1 0]\n",
      " [0 1 0]\n",
      " [0 1 0]\n",
      " ...\n",
      " [0 0 1]\n",
      " [0 1 0]\n",
      " [1 0 0]]\n",
      "df_test       label\n",
      "2993      0\n",
      "1829      1\n",
      "370       1\n",
      "1857      1\n",
      "399      -1\n",
      "...     ...\n",
      "1487      1\n",
      "2467      0\n",
      "3036      0\n",
      "2821      0\n",
      "3201      0\n",
      "\n",
      "[692 rows x 1 columns]\n",
      "df_test (692, 1)\n",
      "y_train_prob [[0.02 0.91 0.07]\n",
      " [0.   0.98 0.02]\n",
      " [0.16 0.82 0.02]\n",
      " ...\n",
      " [0.02 0.23 0.75]\n",
      " [0.   0.91 0.09]\n",
      " [0.67 0.27 0.06]]\n",
      "y_train_prob (2816, 3)\n",
      "y_test_prob [[0.07 0.63 0.3 ]\n",
      " [0.07 0.35 0.58]\n",
      " [0.04 0.78 0.18]\n",
      " ...\n",
      " [0.03 0.91 0.06]\n",
      " [0.05 0.94 0.01]\n",
      " [0.1  0.6  0.3 ]]\n",
      "y_test_prob (692, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6445|precision=0.6070|recall=0.5932|f1=0.5957|auc=0.8116|aupr=0.6801\n",
      "-----------------------Fold =  4\n",
      "X (3508, 845)\n",
      "x_test [[0.96475346 0.9643078  0.98396746 ... 0.90433029 0.29896292 0.27144442]\n",
      " [0.96466937 0.98231691 0.99492622 ... 0.98984646 0.05083032 0.03861112]\n",
      " [0.9804817  0.96914205 0.999644   ... 0.01146701 0.9452225  0.94530427]\n",
      " ...\n",
      " [0.93620782 0.99478677 0.98800118 ... 0.98206772 0.02118915 0.01974696]\n",
      " [0.95556567 0.98299013 0.99428956 ... 0.00248168 0.92496398 0.94094229]\n",
      " [0.97112326 0.97180881 0.99463295 ... 0.00400382 0.90276721 0.91585766]]\n",
      "x_test.shape (694, 845)\n",
      "class_arr [[0 1 0]\n",
      " [1 0 0]\n",
      " [0 1 0]\n",
      " ...\n",
      " [1 0 0]\n",
      " [0 1 0]\n",
      " [0 1 0]]\n",
      "df_test       label\n",
      "2032      1\n",
      "3163     -1\n",
      "455       0\n",
      "606      -1\n",
      "2966      0\n",
      "...     ...\n",
      "1822      1\n",
      "2451     -1\n",
      "277       1\n",
      "372       1\n",
      "2498      0\n",
      "\n",
      "[694 rows x 1 columns]\n",
      "df_test (694, 1)\n",
      "y_train_prob [[0.02 0.94 0.04]\n",
      " [0.69 0.09 0.22]\n",
      " [0.04 0.95 0.01]\n",
      " ...\n",
      " [0.68 0.21 0.11]\n",
      " [0.02 0.94 0.04]\n",
      " [0.01 0.94 0.05]]\n",
      "y_train_prob (2814, 3)\n",
      "y_test_prob [[0.65 0.27 0.08]\n",
      " [0.33 0.65 0.02]\n",
      " [0.31 0.66 0.03]\n",
      " ...\n",
      " [0.04 0.65 0.31]\n",
      " [0.5  0.44 0.06]\n",
      " [0.07 0.81 0.12]]\n",
      "y_test_prob (694, 3)\n",
      "Train:\n",
      "acc=1.0000|precision=1.0000|recall=1.0000|f1=1.0000|auc=1.0000|aupr=1.0000\n",
      "Test:\n",
      "acc=0.6398|precision=0.5991|recall=0.5930|f1=0.5942|auc=0.8056|aupr=0.6653\n"
     ]
    }
   ],
   "source": [
    "#directory = 'C:/Users/Administrator/Desktop/MDA-GCNFTG-main/data'\n",
    "directory='C:/Users/Administrator/Desktop/图采样data/last data'\n",
    "#for isbalance in [False]:\n",
    "for isbalance in [True]:\n",
    "#for isbalance in [True, False]:\n",
    "    \n",
    "    ID, IM, dtp, mirna_ids, disease_ids, mirna_test_num, disease_test_num, samples = obtain_data(directory, isbalance)\n",
    "    for task in ['Tp', 'Tm', 'Td']:\n",
    "    #for task in ['Tp']:\n",
    "        \n",
    "        print('========== isbalance = {} | task = {}'.format(isbalance, task))\n",
    "        \n",
    "        if task == 'Tp':\n",
    "            train_index_all, test_index_all, train_id_all, test_id_all = generate_task_Tp_train_test_idx(samples)\n",
    "            \n",
    "        elif task == 'Tm':\n",
    "            item = 'Drug'\n",
    "            ids = mirna_ids\n",
    "            train_index_all, test_index_all, train_id_all, test_id_all = generate_task_Tm_Td_train_test_idx(item, ids, dtp)\n",
    "\n",
    "        elif task == 'Td':\n",
    "            item = 'Mutation'\n",
    "            ids = disease_ids\n",
    "            train_index_all, test_index_all, train_id_all, test_id_all = generate_task_Tm_Td_train_test_idx(item, ids, dtp)\n",
    "\n",
    "        ys_train, metrics_train, ys_test, metrics_test = run_rf(train_index_all, test_index_all, samples)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def precision(y_true, y_pred):\n",
    "    # Calculates the precision\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "    precision = true_positives / (predicted_positives + K.epsilon())\n",
    "    return precision\n",
    "\n",
    "\n",
    "def recall(y_true, y_pred):\n",
    "    # Calculates the recall\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "    recall = true_positives / (possible_positives + K.epsilon())\n",
    "    return recall\n",
    "\n",
    "def fbeta_score(y_true, y_pred, beta=1):\n",
    "    # Calculates the F score, the weighted harmonic mean of precision and recall.\n",
    "\n",
    "    if beta < 0:\n",
    "        raise ValueError('The lowest choosable beta is zero (only precision).')\n",
    "        \n",
    "    # If there are no true positives, fix the F score at 0 like sklearn.\n",
    "    if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:\n",
    "        return 0\n",
    "\n",
    "    p = precision(y_true, y_pred)\n",
    "    r = recall(y_true, y_pred)\n",
    "    bb = beta ** 2\n",
    "    fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())\n",
    "    return fbeta_score\n",
    "\n",
    "def f1(y_true, y_pred):\n",
    "    # Calculates the f-measure, the harmonic mean of precision and recall.\n",
    "    return fbeta_score(y_true, y_pred, beta=1)\n",
    "\n",
    "\n",
    "def transfer(y_pred):\n",
    "    return [[0,1][x>0.5] for x in y_pred.reshape(-1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 607,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Model(x):\n",
    "    model = Sequential()\n",
    "    model.add(Dense(1024, activation='elu', input_shape=(x.shape[1],)))\n",
    "    model.add(Dense(512, activation='elu'))\n",
    "    model.add(Dense(256, activation='relu'))\n",
    "    model.add(Dense(64, activation='relu'))\n",
    "    model.add(Dense(1, activation='sigmoid'))\n",
    "    optimizer = Adam(lr = 0.0001)\n",
    "    model.compile(optimizer = optimizer, loss = binary_crossentropy, metrics=[binary_accuracy, f1, recall, precision])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 608,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_dnn(train_index_all, test_index_all, samples):\n",
    "    \n",
    "    fold = 0\n",
    "    for train_idx, test_idx in zip(train_index_all, test_index_all):\n",
    "        print('----------------------- Fold = ', str(fold))\n",
    "\n",
    "        X = samples.iloc[:, 3:]\n",
    "        y = samples['label']\n",
    "\n",
    "        scaler = preprocessing.MinMaxScaler().fit(X.iloc[train_idx,:])\n",
    "        X = scaler.transform(X)\n",
    "\n",
    "        x_train, y_train = X[train_idx], y[train_idx]\n",
    "        x_test, y_test = X[test_idx], y[test_idx]\n",
    "\n",
    "        model = Model(x_train)\n",
    "        early_stopping = EarlyStopping(monitor='val_loss', patience = 50)\n",
    "        #model.fit(x_train, y_train, epochs =500, batch_size = 256, validation_data=(x_test, y_test), callbacks=[early_stopping],verbose = 2)\n",
    "        model.fit(x_train, y_train, epochs =100, batch_size = 512, validation_data=(x_test, y_test), callbacks=[early_stopping],verbose = 2)\n",
    "\n",
    "        y_train_pred, y_test_pred = model.predict(x_train, verbose = 0), model.predict(x_test, verbose = 0)\n",
    "        y_train_prob, y_test_prob = model.predict_proba(x_train), model.predict_proba(x_test)\n",
    "\n",
    "        if len(Counter(y_train_pred.reshape(-1))) > 2: \n",
    "            y_train_pred = transfer(y_train_pred)\n",
    "        else:\n",
    "            print(Counter(y_train_pred.reshape(-1)))\n",
    "        if len(Counter(y_test_pred.reshape(-1))) > 2: \n",
    "            y_test_pred = transfer(y_test_pred)\n",
    "        else:\n",
    "            print(Counter(y_test_pred.reshape(-1)))\n",
    "\n",
    "        performances_train = performances(y_train, y_train_pred, y_train_prob)\n",
    "        performances_test = performances(y_test, y_test_pred, y_test_prob)\n",
    "        #print('y_train',y_train)\n",
    "        #print('y_train_pred',y_train_pred)\n",
    "        #print('y_train_prob',y_train_prob)\n",
    "        \n",
    "        \n",
    "        fold += 1\n",
    "    \n",
    "    return y_train_pred, y_test_pred, y_train_prob, y_test_prob, performances_train, performances_test"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DNN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 610,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "5it [00:00, 418.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Drug = 184 | Mutation = 661\n",
      "# Test: Drug = 36 | Mutation = 132\n",
      "========== isbalance = True | task = Tp\n",
      "-------Fold  0\n",
      "# Pairs: Train = 2806 | Test = 702\n",
      "-------Fold  1\n",
      "# Pairs: Train = 2806 | Test = 702\n",
      "-------Fold  2\n",
      "# Pairs: Train = 2806 | Test = 702\n",
      "-------Fold  3\n",
      "# Pairs: Train = 2807 | Test = 701\n",
      "-------Fold  4\n",
      "# Pairs: Train = 2807 | Test = 701\n",
      "----------------------- Fold =  0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2806 samples, validate on 702 samples\n",
      "Epoch 1/100\n",
      " - 3s - loss: 0.1217 - binary_accuracy: 0.4665 - f1: 0.0598 - recall: 0.1032 - precision: 0.0421 - val_loss: -5.5562e-02 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 2/100\n",
      " - 0s - loss: -2.5212e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.0705e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 3/100\n",
      " - 0s - loss: -4.1455e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.3480e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 4/100\n",
      " - 0s - loss: -4.7181e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.0109e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 5/100\n",
      " - 0s - loss: -4.6109e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -9.5364e-02 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 6/100\n",
      " - 0s - loss: -4.6085e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -9.7256e-02 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 7/100\n",
      " - 0s - loss: -4.6273e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.0701e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 8/100\n",
      " - 0s - loss: -4.6731e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.2142e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 9/100\n",
      " - 0s - loss: -4.7235e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.3189e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 10/100\n",
      " - 0s - loss: -4.7262e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.2020e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 11/100\n",
      " - 0s - loss: -4.7335e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.2626e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 12/100\n",
      " - 0s - loss: -4.7400e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.3149e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 13/100\n",
      " - 0s - loss: -4.7455e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.1995e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 14/100\n",
      " - 0s - loss: -4.7496e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.2202e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 15/100\n",
      " - 0s - loss: -4.7672e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.4190e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 16/100\n",
      " - 0s - loss: -4.8073e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.5122e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 17/100\n",
      " - 0s - loss: -4.8263e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.5767e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 18/100\n",
      " - 0s - loss: -4.8520e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.4469e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 19/100\n",
      " - 0s - loss: -4.8540e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.5081e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 20/100\n",
      " - 0s - loss: -4.9423e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.8068e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 21/100\n",
      " - 0s - loss: -4.7918e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.9257e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 22/100\n",
      " - 0s - loss: -4.9595e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.4843e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 23/100\n",
      " - 0s - loss: -4.9249e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.7929e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 24/100\n",
      " - 0s - loss: -5.0398e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -2.2174e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 25/100\n",
      " - 0s - loss: -5.1473e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -1.9740e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 26/100\n",
      " - 0s - loss: -5.1805e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -2.4538e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 27/100\n",
      " - 0s - loss: -5.0463e-01 - binary_accuracy: 0.4922 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -2.4565e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 28/100\n",
      " - 0s - loss: -5.3565e-01 - binary_accuracy: 0.4925 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -2.1897e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 29/100\n",
      " - 0s - loss: -5.3381e-01 - binary_accuracy: 0.4922 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -2.7648e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0120 - val_recall: 0.0062 - val_precision: 0.2707\n",
      "Epoch 30/100\n",
      " - 0s - loss: -5.4240e-01 - binary_accuracy: 0.4907 - f1: 0.0144 - recall: 0.0074 - precision: 0.2737 - val_loss: -2.7334e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0120 - val_recall: 0.0062 - val_precision: 0.2707\n",
      "Epoch 31/100\n",
      " - 0s - loss: -5.3993e-01 - binary_accuracy: 0.4922 - f1: 0.0029 - recall: 0.0015 - precision: 0.1825 - val_loss: -2.6137e-01 - val_binary_accuracy: 0.5242 - val_f1: 0.0120 - val_recall: 0.0062 - val_precision: 0.2707\n",
      "Epoch 32/100\n",
      " - 0s - loss: -5.7732e-01 - binary_accuracy: 0.4904 - f1: 0.0094 - recall: 0.0049 - precision: 0.1095 - val_loss: -3.5070e-01 - val_binary_accuracy: 0.5185 - val_f1: 0.0113 - val_recall: 0.0062 - val_precision: 0.0677\n",
      "Epoch 33/100\n",
      " - 0s - loss: -5.9005e-01 - binary_accuracy: 0.4922 - f1: 0.0088 - recall: 0.0045 - precision: 0.3649 - val_loss: -3.0553e-01 - val_binary_accuracy: 0.5228 - val_f1: 0.0118 - val_recall: 0.0062 - val_precision: 0.1353\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34/100\n",
      " - 0s - loss: -5.9863e-01 - binary_accuracy: 0.4929 - f1: 0.0137 - recall: 0.0071 - precision: 0.3784 - val_loss: -3.6815e-01 - val_binary_accuracy: 0.5185 - val_f1: 0.0113 - val_recall: 0.0062 - val_precision: 0.0677\n",
      "Epoch 35/100\n",
      " - 0s - loss: -5.9322e-01 - binary_accuracy: 0.4914 - f1: 0.0086 - recall: 0.0044 - precision: 0.1521 - val_loss: -3.5748e-01 - val_binary_accuracy: 0.5214 - val_f1: 0.0343 - val_recall: 0.0183 - val_precision: 0.2726\n",
      "Epoch 36/100\n",
      " - 0s - loss: -6.1011e-01 - binary_accuracy: 0.4907 - f1: 0.0335 - recall: 0.0176 - precision: 0.3932 - val_loss: -3.9172e-01 - val_binary_accuracy: 0.5228 - val_f1: 0.0347 - val_recall: 0.0183 - val_precision: 0.3333\n",
      "Epoch 37/100\n",
      " - 0s - loss: -6.2125e-01 - binary_accuracy: 0.4918 - f1: 0.0121 - recall: 0.0063 - precision: 0.1825 - val_loss: -3.4458e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0118 - val_recall: 0.0062 - val_precision: 0.1353\n",
      "Epoch 38/100\n",
      " - 0s - loss: -5.8164e-01 - binary_accuracy: 0.4929 - f1: 0.0032 - recall: 0.0016 - precision: 0.1825 - val_loss: -2.9941e-01 - val_binary_accuracy: 0.5285 - val_f1: 0.0120 - val_recall: 0.0062 - val_precision: 0.2707\n",
      "Epoch 39/100\n",
      " - 0s - loss: -6.0791e-01 - binary_accuracy: 0.4922 - f1: 0.0214 - recall: 0.0111 - precision: 0.4038 - val_loss: -4.0522e-01 - val_binary_accuracy: 0.5214 - val_f1: 0.0343 - val_recall: 0.0183 - val_precision: 0.2726\n",
      "Epoch 40/100\n",
      " - 0s - loss: -6.3342e-01 - binary_accuracy: 0.4922 - f1: 0.0201 - recall: 0.0104 - precision: 0.4246 - val_loss: -4.3529e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0352 - val_recall: 0.0183 - val_precision: 0.5764\n",
      "Epoch 41/100\n",
      " - 0s - loss: -6.1013e-01 - binary_accuracy: 0.4936 - f1: 0.0080 - recall: 0.0041 - precision: 0.2129 - val_loss: -4.0530e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 42/100\n",
      " - 0s - loss: -6.2810e-01 - binary_accuracy: 0.4929 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.0895e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 43/100\n",
      " - 0s - loss: -6.4102e-01 - binary_accuracy: 0.4929 - f1: 0.0031 - recall: 0.0016 - precision: 0.1825 - val_loss: -4.2867e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 44/100\n",
      " - 0s - loss: -6.3723e-01 - binary_accuracy: 0.4932 - f1: 0.0057 - recall: 0.0029 - precision: 0.2737 - val_loss: -3.8847e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 45/100\n",
      " - 0s - loss: -6.3283e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.2575e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 46/100\n",
      " - 0s - loss: -6.4607e-01 - binary_accuracy: 0.4929 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.1159e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 47/100\n",
      " - 0s - loss: -6.5101e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.4442e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 48/100\n",
      " - 0s - loss: -6.4787e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.1572e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 49/100\n",
      " - 0s - loss: -6.6631e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.6471e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 50/100\n",
      " - 0s - loss: -6.7642e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.6889e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 51/100\n",
      " - 0s - loss: -6.7514e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.4558e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 52/100\n",
      " - 0s - loss: -6.5439e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -3.7518e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 53/100\n",
      " - 0s - loss: -6.7789e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.7423e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 54/100\n",
      " - 0s - loss: -6.8850e-01 - binary_accuracy: 0.4925 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.0411e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 55/100\n",
      " - 0s - loss: -6.5378e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.0204e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 56/100\n",
      " - 0s - loss: -6.8346e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.5236e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 57/100\n",
      " - 0s - loss: -6.6541e-01 - binary_accuracy: 0.4929 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -3.7175e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 58/100\n",
      " - 0s - loss: -6.4333e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -3.6920e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 59/100\n",
      " - 0s - loss: -6.7359e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.8943e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 60/100\n",
      " - 0s - loss: -6.3449e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.3765e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 61/100\n",
      " - 0s - loss: -6.4611e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -3.7796e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 62/100\n",
      " - 0s - loss: -6.6192e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.8590e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 63/100\n",
      " - 0s - loss: -6.9406e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.8332e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 64/100\n",
      " - 0s - loss: -6.9517e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.9342e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 65/100\n",
      " - 0s - loss: -6.8302e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.4065e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 66/100\n",
      " - 0s - loss: -7.0284e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -5.0353e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 67/100\n",
      " - 0s - loss: -7.0778e-01 - binary_accuracy: 0.4939 - f1: 0.0087 - recall: 0.0044 - precision: 0.4562 - val_loss: -4.5907e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 68/100\n",
      " - 0s - loss: -6.8648e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.6883e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 69/100\n",
      " - 0s - loss: -6.8094e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.6649e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 70/100\n",
      " - 0s - loss: -7.0795e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.8995e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 71/100\n",
      " - 0s - loss: -7.1525e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.8724e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 72/100\n",
      " - 0s - loss: -7.1501e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.6046e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 73/100\n",
      " - 0s - loss: -7.1684e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.8241e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 74/100\n",
      " - 0s - loss: -7.1771e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -5.1447e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 75/100\n",
      " - 0s - loss: -7.0536e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -5.1306e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 76/100\n",
      " - 0s - loss: -7.1692e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.5986e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 77/100\n",
      " - 0s - loss: -7.1689e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.7593e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 78/100\n",
      " - 0s - loss: -7.2346e-01 - binary_accuracy: 0.4936 - f1: 0.0030 - recall: 0.0015 - precision: 0.1825 - val_loss: -4.8651e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 79/100\n",
      " - 0s - loss: -6.8147e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -3.9943e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 80/100\n",
      " - 0s - loss: -7.2100e-01 - binary_accuracy: 0.4936 - f1: 0.0029 - recall: 0.0014 - precision: 0.1825 - val_loss: -4.7764e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 81/100\n",
      " - 0s - loss: -6.9961e-01 - binary_accuracy: 0.4939 - f1: 0.0060 - recall: 0.0030 - precision: 0.3649 - val_loss: -4.4066e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 82/100\n",
      " - 0s - loss: -7.3193e-01 - binary_accuracy: 0.4936 - f1: 0.0033 - recall: 0.0017 - precision: 0.0877 - val_loss: -4.8752e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 83/100\n",
      " - 0s - loss: -7.2439e-01 - binary_accuracy: 0.4936 - f1: 0.0029 - recall: 0.0015 - precision: 0.1825 - val_loss: -4.6989e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 84/100\n",
      " - 0s - loss: -7.3631e-01 - binary_accuracy: 0.4936 - f1: 0.0093 - recall: 0.0047 - precision: 0.2737 - val_loss: -4.7867e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 85/100\n",
      " - 0s - loss: -7.3585e-01 - binary_accuracy: 0.4936 - f1: 0.0067 - recall: 0.0034 - precision: 0.2737 - val_loss: -5.1673e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 86/100\n",
      " - 0s - loss: -7.0064e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.9340e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 87/100\n",
      " - 0s - loss: -7.2544e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.8723e-01 - val_binary_accuracy: 0.5242 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 88/100\n",
      " - 0s - loss: -7.3497e-01 - binary_accuracy: 0.4936 - f1: 0.0062 - recall: 0.0031 - precision: 0.3649 - val_loss: -4.9660e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 89/100\n",
      " - 0s - loss: -7.3986e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -4.7792e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 90/100\n",
      " - 0s - loss: -7.2492e-01 - binary_accuracy: 0.4954 - f1: 0.0201 - recall: 0.0103 - precision: 0.5895 - val_loss: -5.5319e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 91/100\n",
      " - 0s - loss: -7.3620e-01 - binary_accuracy: 0.4936 - f1: 0.0028 - recall: 0.0014 - precision: 0.1825 - val_loss: -4.3794e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 92/100\n",
      " - 0s - loss: -6.9912e-01 - binary_accuracy: 0.4936 - f1: 0.0028 - recall: 0.0014 - precision: 0.1825 - val_loss: -4.9898e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 93/100\n",
      " - 0s - loss: -7.1733e-01 - binary_accuracy: 0.4932 - f1: 0.0000e+00 - recall: 0.0000e+00 - precision: 0.0000e+00 - val_loss: -3.7529e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 94/100\n",
      " - 0s - loss: -7.1631e-01 - binary_accuracy: 0.4939 - f1: 0.0086 - recall: 0.0044 - precision: 0.3649 - val_loss: -5.4162e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0347 - val_recall: 0.0182 - val_precision: 0.3647\n",
      "Epoch 95/100\n",
      " - 0s - loss: -7.3188e-01 - binary_accuracy: 0.4939 - f1: 0.0314 - recall: 0.0162 - precision: 0.5018 - val_loss: -4.6795e-01 - val_binary_accuracy: 0.5242 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 96/100\n",
      " - 0s - loss: -7.1235e-01 - binary_accuracy: 0.4939 - f1: 0.0058 - recall: 0.0029 - precision: 0.1825 - val_loss: -4.0427e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 97/100\n",
      " - 0s - loss: -7.2724e-01 - binary_accuracy: 0.4936 - f1: 0.0026 - recall: 0.0013 - precision: 0.1825 - val_loss: -5.2034e-01 - val_binary_accuracy: 0.5242 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 98/100\n",
      " - 0s - loss: -7.5129e-01 - binary_accuracy: 0.4936 - f1: 0.0057 - recall: 0.0029 - precision: 0.2737 - val_loss: -4.3883e-01 - val_binary_accuracy: 0.5271 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 99/100\n",
      " - 0s - loss: -7.1830e-01 - binary_accuracy: 0.4936 - f1: 0.0032 - recall: 0.0016 - precision: 0.1825 - val_loss: -5.0525e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0000e+00 - val_recall: 0.0000e+00 - val_precision: 0.0000e+00\n",
      "Epoch 100/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " - 0s - loss: -7.4783e-01 - binary_accuracy: 0.4936 - f1: 0.0088 - recall: 0.0045 - precision: 0.3649 - val_loss: -5.3953e-01 - val_binary_accuracy: 0.5256 - val_f1: 0.0119 - val_recall: 0.0061 - val_precision: 0.2431\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\ipykernel_launcher.py:38: DeprecationWarning: In future, it will be an error for 'np.bool_' scalars to be interpreted as an index\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "multi_class must be in ('ovo', 'ovr')",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_3860/2509351549.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     23\u001b[0m             \u001b[0mtrain_index_all\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_index_all\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtrain_id_all\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_id_all\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mgenerate_task_Tm_Td_train_test_idx\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mids\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtp\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     24\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 25\u001b[1;33m         \u001b[0my_train_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train_prob\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test_prob\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mperformances_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mperformances_test\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mrun_dnn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_index_all\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtest_index_all\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msamples\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_3860/4227059911.py\u001b[0m in \u001b[0;36mrun_dnn\u001b[1;34m(train_index_all, test_index_all, samples)\u001b[0m\n\u001b[0;32m     31\u001b[0m             \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCounter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_test_pred\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m-\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     32\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 33\u001b[1;33m         \u001b[0mperformances_train\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mperformances\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train_prob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     34\u001b[0m         \u001b[0mperformances_test\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mperformances\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_test\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test_pred\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_test_prob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     35\u001b[0m         \u001b[1;31m#print('y_train',y_train)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Local\\Temp/ipykernel_3860/3230708444.py\u001b[0m in \u001b[0;36mperformances\u001b[1;34m(y_true, y_pred, y_prob)\u001b[0m\n\u001b[0;32m     74\u001b[0m     \u001b[0mf1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m2\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mprecision\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mrecall\u001b[0m \u001b[1;33m/\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mprecision\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mrecall\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     75\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 76\u001b[1;33m     \u001b[0mroc_auc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mroc_auc_score\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_prob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     77\u001b[0m     \u001b[0mprec\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mreca\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mprecision_recall_curve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my_true\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_prob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     78\u001b[0m     \u001b[0maupr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mauc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mreca\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mprec\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\sklearn\\utils\\validation.py\u001b[0m in \u001b[0;36minner_f\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     61\u001b[0m             \u001b[0mextra_args\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mall_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     62\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mextra_args\u001b[0m \u001b[1;33m<=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 63\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     64\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     65\u001b[0m             \u001b[1;31m# extra_args > 0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\envs\\GCNFTG\\lib\\site-packages\\sklearn\\metrics\\_ranking.py\u001b[0m in \u001b[0;36mroc_auc_score\u001b[1;34m(y_true, y_score, average, sample_weight, max_fpr, multi_class, labels)\u001b[0m\n\u001b[0;32m    534\u001b[0m                              \"instead\".format(max_fpr))\n\u001b[0;32m    535\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmulti_class\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'raise'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 536\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"multi_class must be in ('ovo', 'ovr')\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    537\u001b[0m         return _multiclass_roc_auc_score(y_true, y_score, labels,\n\u001b[0;32m    538\u001b[0m                                          multi_class, average, sample_weight)\n",
      "\u001b[1;31mValueError\u001b[0m: multi_class must be in ('ovo', 'ovr')"
     ]
    }
   ],
   "source": [
    "#directory = 'C:/Users/Administrator/Desktop/MDA-GCNFTG-main/data'\n",
    "directory='C:/Users/Administrator/Desktop/图采样data/last data'\n",
    "for isbalance in [True]:\n",
    "#for isbalance in [True, False]:\n",
    "    \n",
    "    ID, IM, dtp, mirna_ids, disease_ids, mirna_test_num, disease_test_num, samples = obtain_data(directory, \n",
    "                                                                                                 isbalance)\n",
    "    #for task in ['Tp', 'Tm', 'Td']:\n",
    "    for task in ['Tp']:   \n",
    "        print('========== isbalance = {} | task = {}'.format(isbalance, task))\n",
    "        \n",
    "        if task == 'Tp':\n",
    "            train_index_all, test_index_all, train_id_all, test_id_all = generate_task_Tp_train_test_idx(samples)\n",
    "            \n",
    "        elif task == 'Tm':\n",
    "            item = 'miRNA'\n",
    "            ids = mirna_ids\n",
    "            train_index_all, test_index_all, train_id_all, test_id_all = generate_task_Tm_Td_train_test_idx(item, ids, dtp)\n",
    "\n",
    "        elif task == 'Td':\n",
    "            item = 'disease'\n",
    "            ids = disease_ids\n",
    "            train_index_all, test_index_all, train_id_all, test_id_all = generate_task_Tm_Td_train_test_idx(item, ids, dtp)\n",
    "\n",
    "        y_train_pred, y_test_pred, y_train_prob, y_test_prob, performances_train, performances_test = run_dnn(train_index_all, test_index_all, samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
